{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "PyTorch: optim\n",
    "--------------\n",
    "\n",
    "A fully-connected ReLU network with one hidden layer, trained to predict y from x\n",
    "by minimizing squared Euclidean distance.\n",
    "\n",
    "This implementation uses the nn package from PyTorch to build the network.\n",
    "\n",
    "<strong style=\"color:red\">Rather than manually updating the weights of the model as we have been doing.</strong>,\n",
    "we use the<strong style=\"color:red\"> optim package.</strong> to define an Optimizer that will <strong style=\"color:green\">update the weights for us</strong>. \n",
    "\n",
    "The optim package defines many optimization algorithms that are commonly used for deep learning, including SGD+momentum, RMSProp, Adam, etc.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h1 style=\"background-image: linear-gradient( 135deg, #ABDCFF 10%, #0396FF 100%);\"> Orinal Tutorial code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 682.8252563476562\n",
      "1 666.3059692382812\n",
      "2 650.2415161132812\n",
      "3 634.7083740234375\n",
      "4 619.6635131835938\n",
      "5 605.11474609375\n",
      "6 590.9830932617188\n",
      "7 577.2006225585938\n",
      "8 563.7789916992188\n",
      "9 550.6875\n",
      "10 537.9588623046875\n",
      "11 525.55322265625\n",
      "12 513.4520874023438\n",
      "13 501.6575012207031\n",
      "14 490.21295166015625\n",
      "15 479.06243896484375\n",
      "16 468.2439880371094\n",
      "17 457.754150390625\n",
      "18 447.5335388183594\n",
      "19 437.612548828125\n",
      "20 427.9945373535156\n",
      "21 418.6335144042969\n",
      "22 409.53851318359375\n",
      "23 400.7070617675781\n",
      "24 392.14263916015625\n",
      "25 383.79864501953125\n",
      "26 375.6753234863281\n",
      "27 367.7168273925781\n",
      "28 359.9151916503906\n",
      "29 352.2444763183594\n",
      "30 344.73577880859375\n",
      "31 337.41595458984375\n",
      "32 330.25653076171875\n",
      "33 323.2464904785156\n",
      "34 316.3948669433594\n",
      "35 309.6763000488281\n",
      "36 303.0809326171875\n",
      "37 296.64385986328125\n",
      "38 290.3456726074219\n",
      "39 284.16912841796875\n",
      "40 278.12176513671875\n",
      "41 272.1761169433594\n",
      "42 266.3402099609375\n",
      "43 260.6186828613281\n",
      "44 255.04672241210938\n",
      "45 249.58338928222656\n",
      "46 244.2247314453125\n",
      "47 238.97825622558594\n",
      "48 233.82589721679688\n",
      "49 228.77139282226562\n",
      "50 223.80184936523438\n",
      "51 218.91705322265625\n",
      "52 214.12965393066406\n",
      "53 209.42794799804688\n",
      "54 204.79434204101562\n",
      "55 200.23097229003906\n",
      "56 195.75881958007812\n",
      "57 191.36871337890625\n",
      "58 187.0558624267578\n",
      "59 182.81344604492188\n",
      "60 178.64283752441406\n",
      "61 174.54544067382812\n",
      "62 170.51710510253906\n",
      "63 166.5652618408203\n",
      "64 162.6821746826172\n",
      "65 158.87681579589844\n",
      "66 155.13726806640625\n",
      "67 151.4616241455078\n",
      "68 147.86154174804688\n",
      "69 144.32728576660156\n",
      "70 140.85548400878906\n",
      "71 137.43577575683594\n",
      "72 134.07867431640625\n",
      "73 130.78805541992188\n",
      "74 127.55580139160156\n",
      "75 124.39280700683594\n",
      "76 121.28721618652344\n",
      "77 118.23833465576172\n",
      "78 115.24642181396484\n",
      "79 112.31523132324219\n",
      "80 109.43357849121094\n",
      "81 106.61331939697266\n",
      "82 103.85125732421875\n",
      "83 101.15032196044922\n",
      "84 98.50273132324219\n",
      "85 95.90909576416016\n",
      "86 93.37037658691406\n",
      "87 90.88330841064453\n",
      "88 88.4500503540039\n",
      "89 86.07254028320312\n",
      "90 83.743408203125\n",
      "91 81.46209716796875\n",
      "92 79.23502349853516\n",
      "93 77.05784606933594\n",
      "94 74.9243392944336\n",
      "95 72.83770751953125\n",
      "96 70.79986572265625\n",
      "97 68.81156158447266\n",
      "98 66.86528015136719\n",
      "99 64.96256256103516\n",
      "100 63.09862518310547\n",
      "101 61.278114318847656\n",
      "102 59.50270080566406\n",
      "103 57.7703742980957\n",
      "104 56.081626892089844\n",
      "105 54.432960510253906\n",
      "106 52.82505798339844\n",
      "107 51.25471115112305\n",
      "108 49.72077941894531\n",
      "109 48.227256774902344\n",
      "110 46.772979736328125\n",
      "111 45.35440444946289\n",
      "112 43.96977233886719\n",
      "113 42.617431640625\n",
      "114 41.30213165283203\n",
      "115 40.01865005493164\n",
      "116 38.77113723754883\n",
      "117 37.556739807128906\n",
      "118 36.37126541137695\n",
      "119 35.21644592285156\n",
      "120 34.092445373535156\n",
      "121 32.998802185058594\n",
      "122 31.933183670043945\n",
      "123 30.896535873413086\n",
      "124 29.886192321777344\n",
      "125 28.903867721557617\n",
      "126 27.9467716217041\n",
      "127 27.016572952270508\n",
      "128 26.11210060119629\n",
      "129 25.2331485748291\n",
      "130 24.378625869750977\n",
      "131 23.54905891418457\n",
      "132 22.743135452270508\n",
      "133 21.960935592651367\n",
      "134 21.20213508605957\n",
      "135 20.465715408325195\n",
      "136 19.751272201538086\n",
      "137 19.05795669555664\n",
      "138 18.38597869873047\n",
      "139 17.734371185302734\n",
      "140 17.102256774902344\n",
      "141 16.490657806396484\n",
      "142 15.898241996765137\n",
      "143 15.324533462524414\n",
      "144 14.768182754516602\n",
      "145 14.230087280273438\n",
      "146 13.70905876159668\n",
      "147 13.204355239868164\n",
      "148 12.714816093444824\n",
      "149 12.241093635559082\n",
      "150 11.783062934875488\n",
      "151 11.339454650878906\n",
      "152 10.91075611114502\n",
      "153 10.49591064453125\n",
      "154 10.095211029052734\n",
      "155 9.707585334777832\n",
      "156 9.333224296569824\n",
      "157 8.97165584564209\n",
      "158 8.622885704040527\n",
      "159 8.286113739013672\n",
      "160 7.960876941680908\n",
      "161 7.647853374481201\n",
      "162 7.345535755157471\n",
      "163 7.05430269241333\n",
      "164 6.77324104309082\n",
      "165 6.5027008056640625\n",
      "166 6.242269515991211\n",
      "167 5.991109848022461\n",
      "168 5.749329090118408\n",
      "169 5.516572952270508\n",
      "170 5.292239665985107\n",
      "171 5.076523303985596\n",
      "172 4.869076251983643\n",
      "173 4.66934871673584\n",
      "174 4.477324962615967\n",
      "175 4.292777061462402\n",
      "176 4.115259647369385\n",
      "177 3.9445736408233643\n",
      "178 3.780380964279175\n",
      "179 3.622464895248413\n",
      "180 3.47060489654541\n",
      "181 3.324413299560547\n",
      "182 3.1840169429779053\n",
      "183 3.049001693725586\n",
      "184 2.9193837642669678\n",
      "185 2.7946386337280273\n",
      "186 2.6748597621917725\n",
      "187 2.559783458709717\n",
      "188 2.4491946697235107\n",
      "189 2.3429956436157227\n",
      "190 2.2409887313842773\n",
      "191 2.143040180206299\n",
      "192 2.048978328704834\n",
      "193 1.9586602449417114\n",
      "194 1.871944785118103\n",
      "195 1.788757562637329\n",
      "196 1.7089349031448364\n",
      "197 1.632333755493164\n",
      "198 1.5588908195495605\n",
      "199 1.4885112047195435\n",
      "200 1.4209702014923096\n",
      "201 1.3562960624694824\n",
      "202 1.2944270372390747\n",
      "203 1.2351138591766357\n",
      "204 1.1782855987548828\n",
      "205 1.1238632202148438\n",
      "206 1.0717276334762573\n",
      "207 1.0218374729156494\n",
      "208 0.974075436592102\n",
      "209 0.9283733367919922\n",
      "210 0.8846453428268433\n",
      "211 0.8428459167480469\n",
      "212 0.8028298020362854\n",
      "213 0.7645723819732666\n",
      "214 0.7279846668243408\n",
      "215 0.6929947137832642\n",
      "216 0.6595689654350281\n",
      "217 0.6276300549507141\n",
      "218 0.597126841545105\n",
      "219 0.5680041313171387\n",
      "220 0.5401839017868042\n",
      "221 0.5136284828186035\n",
      "222 0.4882884621620178\n",
      "223 0.4641079306602478\n",
      "224 0.44102954864501953\n",
      "225 0.41901856660842896\n",
      "226 0.3980129063129425\n",
      "227 0.37800073623657227\n",
      "228 0.3589247465133667\n",
      "229 0.34073954820632935\n",
      "230 0.3234022557735443\n",
      "231 0.30690112709999084\n",
      "232 0.2912074029445648\n",
      "233 0.2762228548526764\n",
      "234 0.2619752585887909\n",
      "235 0.24841871857643127\n",
      "236 0.23552007973194122\n",
      "237 0.22328785061836243\n",
      "238 0.2116633653640747\n",
      "239 0.20061078667640686\n",
      "240 0.1901027262210846\n",
      "241 0.18011827766895294\n",
      "242 0.17063605785369873\n",
      "243 0.1616157442331314\n",
      "244 0.15305957198143005\n",
      "245 0.14493492245674133\n",
      "246 0.13722500205039978\n",
      "247 0.12990331649780273\n",
      "248 0.12295558303594589\n",
      "249 0.11636251956224442\n",
      "250 0.11010758578777313\n",
      "251 0.10417574644088745\n",
      "252 0.09855257719755173\n",
      "253 0.09321760386228561\n",
      "254 0.08816401660442352\n",
      "255 0.08337239176034927\n",
      "256 0.07883074879646301\n",
      "257 0.07452867925167084\n",
      "258 0.07045282423496246\n",
      "259 0.06659211218357086\n",
      "260 0.06293608993291855\n",
      "261 0.059473440051078796\n",
      "262 0.056196801364421844\n",
      "263 0.05309257283806801\n",
      "264 0.050155479460954666\n",
      "265 0.0473758764564991\n",
      "266 0.04474621266126633\n",
      "267 0.04225831478834152\n",
      "268 0.03990494832396507\n",
      "269 0.03767874091863632\n",
      "270 0.03557337448000908\n",
      "271 0.03358319401741028\n",
      "272 0.031700462102890015\n",
      "273 0.02992139756679535\n",
      "274 0.028239887207746506\n",
      "275 0.026650534942746162\n",
      "276 0.02514854073524475\n",
      "277 0.023731183260679245\n",
      "278 0.022389577701687813\n",
      "279 0.021123314276337624\n",
      "280 0.019927063956856728\n",
      "281 0.018797429278492928\n",
      "282 0.017730621621012688\n",
      "283 0.016723407432436943\n",
      "284 0.015772270038723946\n",
      "285 0.01487424410879612\n",
      "286 0.014026452787220478\n",
      "287 0.013226229697465897\n",
      "288 0.012471025809645653\n",
      "289 0.011758105829358101\n",
      "290 0.011085528880357742\n",
      "291 0.010450810194015503\n",
      "292 0.009851888753473759\n",
      "293 0.009286782704293728\n",
      "294 0.008753347210586071\n",
      "295 0.008250399492681026\n",
      "296 0.0077759455889463425\n",
      "297 0.007328432518988848\n",
      "298 0.006906363647431135\n",
      "299 0.006508256308734417\n",
      "300 0.006132897455245256\n",
      "301 0.005778816994279623\n",
      "302 0.005444975569844246\n",
      "303 0.0051302784122526646\n",
      "304 0.004833480808883905\n",
      "305 0.0045536234974861145\n",
      "306 0.004289851523935795\n",
      "307 0.004041180945932865\n",
      "308 0.0038067384157329798\n",
      "309 0.0035857718903571367\n",
      "310 0.003377511166036129\n",
      "311 0.0031812337692826986\n",
      "312 0.002996206283569336\n",
      "313 0.0028217954095453024\n",
      "314 0.002657511970028281\n",
      "315 0.002502621151506901\n",
      "316 0.002356701996177435\n",
      "317 0.0022192182950675488\n",
      "318 0.0020896063651889563\n",
      "319 0.001967627089470625\n",
      "320 0.0018527292413637042\n",
      "321 0.0017444868572056293\n",
      "322 0.0016425002831965685\n",
      "323 0.0015464340103790164\n",
      "324 0.00145593355409801\n",
      "325 0.0013706880854442716\n",
      "326 0.0012904059840366244\n",
      "327 0.0012147806119173765\n",
      "328 0.0011435317574068904\n",
      "329 0.0010764416074380279\n",
      "330 0.0010132422903552651\n",
      "331 0.0009537142468616366\n",
      "332 0.0008976613753475249\n",
      "333 0.0008448574808426201\n",
      "334 0.0007951356237754226\n",
      "335 0.0007483229273930192\n",
      "336 0.0007042193319648504\n",
      "337 0.0006627115071751177\n",
      "338 0.0006235965993255377\n",
      "339 0.0005867805448360741\n",
      "340 0.0005521114799194038\n",
      "341 0.0005194804398342967\n",
      "342 0.000488760182633996\n",
      "343 0.00045983269228599966\n",
      "344 0.00043258932419121265\n",
      "345 0.0004069447168149054\n",
      "346 0.00038280829903669655\n",
      "347 0.0003601002099458128\n",
      "348 0.0003387008036952466\n",
      "349 0.0003185663081239909\n",
      "350 0.00029961115797050297\n",
      "351 0.0002817816275637597\n",
      "352 0.00026500035892240703\n",
      "353 0.00024919622228480875\n",
      "354 0.00023431771842297167\n",
      "355 0.00022032308334019035\n",
      "356 0.00020715944992844015\n",
      "357 0.00019476488523650914\n",
      "358 0.00018310543964616954\n",
      "359 0.00017214019317179918\n",
      "360 0.00016181875253096223\n",
      "361 0.0001521016238257289\n",
      "362 0.00014295992150437087\n",
      "363 0.00013436628796625882\n",
      "364 0.00012628168042283505\n",
      "365 0.00011867193097714335\n",
      "366 0.00011151717626489699\n",
      "367 0.0001047811601893045\n",
      "368 9.845325257629156e-05\n",
      "369 9.249860886484385e-05\n",
      "370 8.689780224813148e-05\n",
      "371 8.16343745100312e-05\n",
      "372 7.668361649848521e-05\n",
      "373 7.202160486485809e-05\n",
      "374 6.764382123947144e-05\n",
      "375 6.352530908770859e-05\n",
      "376 5.9656282246578485e-05\n",
      "377 5.601952943834476e-05\n",
      "378 5.260053512756713e-05\n",
      "379 4.938696656608954e-05\n",
      "380 4.6365086745936424e-05\n",
      "381 4.3523839849513024e-05\n",
      "382 4.085967884748243e-05\n",
      "383 3.835059033008292e-05\n",
      "384 3.599497358663939e-05\n",
      "385 3.3780208468670025e-05\n",
      "386 3.1703351851319894e-05\n",
      "387 2.9748291126452386e-05\n",
      "388 2.7914875317947008e-05\n",
      "389 2.6188421543338336e-05\n",
      "390 2.4568915250711143e-05\n",
      "391 2.304936970176641e-05\n",
      "392 2.1617550373775885e-05\n",
      "393 2.0278870579204522e-05\n",
      "394 1.9018107195734046e-05\n",
      "395 1.7836568076745607e-05\n",
      "396 1.6723639419069514e-05\n",
      "397 1.568011248309631e-05\n",
      "398 1.470384177082451e-05\n",
      "399 1.3784347174805589e-05\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "400 1.2921124834974762e-05\n",
      "401 1.2110701391065959e-05\n",
      "402 1.1349991837050766e-05\n",
      "403 1.0638404091878328e-05\n",
      "404 9.96866583591327e-06\n",
      "405 9.339854841527995e-06\n",
      "406 8.751857421884779e-06\n",
      "407 8.19921842776239e-06\n",
      "408 7.680937414988875e-06\n",
      "409 7.193659712356748e-06\n",
      "410 6.736984687449876e-06\n",
      "411 6.30889644526178e-06\n",
      "412 5.907951617700746e-06\n",
      "413 5.5324198910966516e-06\n",
      "414 5.179457730264403e-06\n",
      "415 4.8479714678251185e-06\n",
      "416 4.537897439149674e-06\n",
      "417 4.2478386603761464e-06\n",
      "418 3.973785169364419e-06\n",
      "419 3.719066171470331e-06\n",
      "420 3.4795232295437017e-06\n",
      "421 3.2558061775489477e-06\n",
      "422 3.044882305403007e-06\n",
      "423 2.8484348604251863e-06\n",
      "424 2.6637344490154646e-06\n",
      "425 2.491130317139323e-06\n",
      "426 2.329381459276192e-06\n",
      "427 2.177650230805739e-06\n",
      "428 2.0360484995762818e-06\n",
      "429 1.9024266748601804e-06\n",
      "430 1.7789180901672808e-06\n",
      "431 1.6623555438854964e-06\n",
      "432 1.5532614270341583e-06\n",
      "433 1.4511902008962352e-06\n",
      "434 1.3558548062064801e-06\n",
      "435 1.266283334189211e-06\n",
      "436 1.1829897630377673e-06\n",
      "437 1.1043914582842262e-06\n",
      "438 1.0316183534087031e-06\n",
      "439 9.628911357140169e-07\n",
      "440 8.990199944491906e-07\n",
      "441 8.394488872909278e-07\n",
      "442 7.830726076463179e-07\n",
      "443 7.309129159693839e-07\n",
      "444 6.820350790803786e-07\n",
      "445 6.361972282320494e-07\n",
      "446 5.927642519054643e-07\n",
      "447 5.53294341898436e-07\n",
      "448 5.159613465366419e-07\n",
      "449 4.811384997083223e-07\n",
      "450 4.4873632987219025e-07\n",
      "451 4.1816372231551213e-07\n",
      "452 3.8985768924248987e-07\n",
      "453 3.6348336607261444e-07\n",
      "454 3.385027298463683e-07\n",
      "455 3.1562240110361017e-07\n",
      "456 2.9396909440038144e-07\n",
      "457 2.736520627877326e-07\n",
      "458 2.5475463871771353e-07\n",
      "459 2.371794067812516e-07\n",
      "460 2.2089793105806166e-07\n",
      "461 2.058139330074482e-07\n",
      "462 1.9135558204652625e-07\n",
      "463 1.7802665297494968e-07\n",
      "464 1.6582033879330993e-07\n",
      "465 1.5430444477715355e-07\n",
      "466 1.434588341453491e-07\n",
      "467 1.3329386661098397e-07\n",
      "468 1.241170508592404e-07\n",
      "469 1.152849478103235e-07\n",
      "470 1.0721971932525776e-07\n",
      "471 9.966036884634377e-08\n",
      "472 9.275237289330107e-08\n",
      "473 8.613525892542384e-08\n",
      "474 7.99797419404058e-08\n",
      "475 7.42878256687618e-08\n",
      "476 6.921294470885186e-08\n",
      "477 6.409183583855338e-08\n",
      "478 5.946651882027254e-08\n",
      "479 5.531222768695443e-08\n",
      "480 5.1412072821221955e-08\n",
      "481 4.778521400794489e-08\n",
      "482 4.427990418776062e-08\n",
      "483 4.105699247247685e-08\n",
      "484 3.813682525333206e-08\n",
      "485 3.544408855304937e-08\n",
      "486 3.285553162868382e-08\n",
      "487 3.045493102149521e-08\n",
      "488 2.82632761638979e-08\n",
      "489 2.6301400168904365e-08\n",
      "490 2.4336916482070592e-08\n",
      "491 2.2532173460376725e-08\n",
      "492 2.0982042769901454e-08\n",
      "493 1.9425158370722784e-08\n",
      "494 1.801711846383114e-08\n",
      "495 1.6691686255398963e-08\n",
      "496 1.547219241615494e-08\n",
      "497 1.4348437105127232e-08\n",
      "498 1.3332413395517051e-08\n",
      "499 1.2380610314721707e-08\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# N is batch size; D_in is input dimension;\n",
    "# H is hidden dimension; D_out is output dimension.\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "# Create random Tensors to hold inputs and outputs, and wrap them in Variables.\n",
    "x = Variable(torch.randn(N, D_in))\n",
    "y = Variable(torch.randn(N, D_out), requires_grad=False)\n",
    "\n",
    "# Use the nn package to define our model and loss function.\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(D_in, H),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(H, D_out),\n",
    ")\n",
    "\n",
    "# loss function\n",
    "loss_fn = torch.nn.MSELoss(size_average=False)\n",
    "\n",
    "# Use the optim package to define an Optimizer that will update the weights of\n",
    "# the model for us. Here we will use Adam; the optim package contains many other\n",
    "# optimization algoriths. The first argument to the Adam constructor tells the\n",
    "# optimizer which Variables it should update.\n",
    "learning_rate = 1e-4\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)\n",
    "for t in range(500):\n",
    "    # Forward pass: compute predicted y by passing x to the model.\n",
    "    y_pred = model(x)\n",
    "\n",
    "    # Compute and print loss.\n",
    "    loss = loss_fn(y_pred, y)\n",
    "    print(t, loss.data[0])\n",
    "\n",
    "    # Before the backward pass, use the optimizer object to zero all of the\n",
    "    # gradients for the variables it will update (which are the learnable\n",
    "    # weights of the model). This is because by default, gradients are\n",
    "    # accumulated in buffers( i.e, not overwritten) whenever .backward()\n",
    "    # is called. Checkout docs of torch.autograd.backward for more details.\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    # Backward pass: compute gradient of the loss with respect to model parameters\n",
    "    loss.backward()\n",
    "\n",
    "    # Calling the step function on an Optimizer makes an update to its parameters\n",
    "    optimizer.step()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
